{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "biLM.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [
        "jDh9T3MZ6ORH",
        "GFsZk5tk3KU1"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python2",
      "display_name": "Python 2"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AshwinDeshpande96/Speech-Generation/blob/master/biLM.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HJuPGQtSjp5l",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "a4ff8227-13ac-44de-e696-cdce3ac58255"
      },
      "source": [
        "from __future__ import print_function\n",
        "#1.2.\n",
        "from google.colab import drive\n",
        "import codecs\n",
        "#1.3.1.\n",
        "from nltk.corpus import state_union\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.tokenize import RegexpTokenizer\n",
        "#1.3.2.\n",
        "from nltk.corpus import stopwords\n",
        "#1.3.3.\n",
        "from nltk.stem import PorterStemmer\n",
        "#1.3.4.\n",
        "from nltk.corpus import wordnet\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "#1.4.\n",
        "import numpy as np\n",
        "from keras.utils.np_utils import to_categorical\n",
        "#1.5.\n",
        "import h5py\n",
        "#3.1.\n",
        "from keras.layers import Input\n",
        "from keras.layers import LSTM\n",
        "from keras.layers import Flatten\n",
        "from keras.layers import concatenate\n",
        "from keras.layers import Dense\n",
        "from keras.models import Model\n",
        "#3.2.\n",
        "from sklearn.model_selection import train_test_split\n",
        "from keras.callbacks import EarlyStopping\n",
        "from keras.callbacks import ModelCheckpoint\n",
        "from keras.callbacks import ReduceLROnPlateau\n",
        "from keras.models import load_model"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "psXhMobHtg5z",
        "colab_type": "text"
      },
      "source": [
        "# 1. Preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jDh9T3MZ6ORH",
        "colab_type": "text"
      },
      "source": [
        "## 1.1. Collection of resources"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hHaxWiAf51IT",
        "colab_type": "code",
        "outputId": "ec17ee3d-8969-4c4b-cf21-5458efb9a7b6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "import nltk\n",
        "nltk.download('punkt')\n",
        "nltk.download('stopwords')\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "nltk.download('wordnet')"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
            "[nltk_data]       date!\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "toJFqo6Q30Md",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "stop_words = set(stopwords.words(\"english\"))\n",
        "porterStemmer = PorterStemmer()\n",
        "lemmatizer = WordNetLemmatizer()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yq-LnYQ38fvo",
        "colab_type": "text"
      },
      "source": [
        "## 1.2. Import Text File"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x243ce7Zj_is",
        "colab_type": "code",
        "outputId": "e2b37d29-f2ff-4e43-9afe-e590985ee7c9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#drive.mount('/content/gdrive', force_remount=True)\n",
        "drive.mount('/content/gdrive')\n",
        "\n",
        "president = 'lbjohnson'\n",
        "file_path = '/content/gdrive/My Drive/Projects/NLP/President Speech/' +president+'_all.txt'\n",
        "\n",
        "raw_text = open(file_path).read()\n",
        "aw_text = raw_text.lower()"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yb7qxwBx9Naz",
        "colab_type": "text"
      },
      "source": [
        "## 1.3. Corpus Filtering"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w6D1a-sfBBEU",
        "colab_type": "text"
      },
      "source": [
        "### 1.3.1. Tokenize"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_CoiRHXgA54q",
        "colab_type": "code",
        "outputId": "af6203d6-9bbd-440e-8910-b65847232f8a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "words = word_tokenize(raw_text)\n",
        "print(\"Number of tokens in text: \", len(words))"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of tokens in text:  282050\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_K4SbqrBcMYk",
        "colab_type": "code",
        "outputId": "4f76604a-1b65-4282-e54b-6cab67ad549b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "tokenizer = RegexpTokenizer(r'\\w+')\n",
        "words = tokenizer.tokenize(raw_text)\n",
        "print(\"Number of tokens in text: \", len(words))"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of tokens in text:  253825\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gqb3jIi86Zch",
        "colab_type": "text"
      },
      "source": [
        "### 1.3.2. Stop Word Removal"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ccEmN66s4xk4",
        "colab_type": "code",
        "outputId": "7251ffbf-f660-4f20-f997-d9907882ab5d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "\n",
        "words_stop  = [w for w in words if w not in stop_words]\n",
        "print(\"Number of tokens remaining: \", len(words_stop))"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of tokens remaining:  133963\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZCasWM5a9c-R",
        "colab_type": "text"
      },
      "source": [
        "### 1.3.3. Stemming"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9KMoQP7V8OFN",
        "colab_type": "code",
        "outputId": "86c84bd6-916d-4150-e9b5-79549501b39a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "\n",
        "words_stem = [porterStemmer.stem(w) for w in words_stop]\n",
        "print(\"Stemmed Word Sample: \", words_stem[:10])"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Stemmed Word Sample:  [u'honor', 'To', u'henri', 'clayon', 'fourth', 'day', u'juli', '1776', u'peopl', u'feebl']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gBhYHhScO9P4",
        "colab_type": "text"
      },
      "source": [
        "### 1.3.4. Lemmatization"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QuC5DVf-UUVz",
        "colab_type": "text"
      },
      "source": [
        "https://www.machinelearningplus.com/nlp/lemmatization-examples-python/"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kF_uLzZMO8GK",
        "colab_type": "code",
        "outputId": "79cef094-7aa4-4e84-c80d-8f936c390495",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "def get_wordnet_pos(word):\n",
        "    \"\"\"Map POS tag to first character lemmatize() accepts\"\"\"\n",
        "    tag = nltk.pos_tag([word])[0][1][0].upper()\n",
        "    tag_dict = {\"J\": wordnet.ADJ,\n",
        "                \"N\": wordnet.NOUN,\n",
        "                \"V\": wordnet.VERB,\n",
        "                \"R\": wordnet.ADV}\n",
        "\n",
        "    return tag_dict.get(tag, wordnet.NOUN)\n",
        "\n",
        "\n",
        "words_lemma = [lemmatizer.lemmatize(w, get_wordnet_pos(w)) for w in words_stop]\n",
        "print('Lemmatized Word Sample: %s\\nSize: %d'%(words_lemma[:10], len(words_lemma)))"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Lemmatized Word Sample: ['Mr', 'Speaker', 'Mr', 'President', 'Members', 'Congress', 'fellow', 'AmericansFor', 'sixth', 'last']\n",
            "Size: 133963\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ccfpRkDhk8Rs",
        "colab_type": "text"
      },
      "source": [
        "## 1.4. Build Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xbwNZBn5k5g-",
        "colab_type": "code",
        "outputId": "b78ccbf9-659f-4f95-bf2e-6eb8b9c943c3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "vocabulary = np.unique(sorted(words_lemma)).astype('str')\n",
        "word_to_int = dict((str(word), i) for i, word  in enumerate(vocabulary))\n",
        "int_to_word = dict((i, str(word)) for i, word  in enumerate(vocabulary))\n",
        "num_words = len(words_lemma)\n",
        "vocab_size = len(vocabulary)\n",
        "print(\"Number of words: %d\\nVocabulary Size: %d\"%(num_words, vocab_size))"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of words: 133963\n",
            "Vocabulary Size: 8073\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q5kBVibBl5nc",
        "colab_type": "code",
        "outputId": "ad0d87ba-8c57-415d-b249-f1ca8bc4f99d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "n = 10\n",
        "X_left = []\n",
        "X_right = []\n",
        "Y = []\n",
        "for i in range(n, num_words - (n+1), 1):\n",
        "    left_in = words_lemma[i-n:i]\n",
        "    right_in = words_lemma[i+1 : i+n+1]\n",
        "    out = words_lemma[i]\n",
        "    X_left.append([word_to_int[str(w)] for w in left_in])\n",
        "    X_right.append([word_to_int[str(w)] for w in right_in])\n",
        "    Y.append(word_to_int[str(out)])\n",
        "\n",
        "n_patterns_left = len(X_left)\n",
        "n_patterns_right = len(X_right)\n",
        "print(\"Left #Patterns: %d\\tX_left shape: %s\"%(n_patterns_left, np.array(X_left).shape,))\n",
        "print(\"Right #Patterns: %d\\tX_right shape: %s\"%(n_patterns_right, np.array(X_right).shape,))\n",
        "\n",
        "\n",
        "##############################################################################################################################\n",
        "\n",
        "# reshape X to be [samples, time steps, features]\n",
        "X_left = np.reshape(X_left, (n_patterns_left, n, 1))\n",
        "X_right = np.reshape(X_right, (n_patterns_right, n, 1))\n",
        "# normalize\n",
        "X_left = X_left / float(num_words)\n",
        "X_right = X_right / float(num_words)\n",
        "print(\"X_left shape: %s\\nX_right shape: %s\"%(X_left.shape, X_right.shape))\n",
        "# one hot encode the output variable\n",
        "Y = to_categorical(Y, num_classes=vocab_size, dtype='float32')\n",
        "print (\"Y shape: \", Y.shape)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Left #Patterns: 133942\tX_left shape: (133942, 10)\n",
            "Right #Patterns: 133942\tX_right shape: (133942, 10)\n",
            "X_left shape: (133942, 10, 1)\n",
            "X_right shape: (133942, 10, 1)\n",
            "Y shape:  (133942, 8073)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GFsZk5tk3KU1",
        "colab_type": "text"
      },
      "source": [
        "## 1.5. Save"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OZZ-Vxy83JbI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "left = h5py.File(\"/content/gdrive/My Drive/Projects/NLP/President Speech/left.hdf5\", \"w\")\n",
        "right = h5py.File(\"/content/gdrive/My Drive/Projects/NLP/President Speech/right.hdf5\", \"w\")\n",
        "out = h5py.File(\"/content/gdrive/My Drive/Projects/NLP/President Speech/out.hdf5\", \"w\")\n",
        "\n",
        "left.create_dataset('dataset_left', data=X_left)\n",
        "right.create_dataset('dataset_right', data=X_right)\n",
        "out.create_dataset('dataset_out', data=Y)\n",
        "\n",
        "left.close()\n",
        "right.close()\n",
        "out.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xY7TncGBYeni",
        "colab_type": "text"
      },
      "source": [
        "# 2. Import Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1P7pFzWGYvts",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "outputId": "d5719809-fa50-4e3c-fb00-fcf8a4b2dfd7"
      },
      "source": [
        "drive.mount('/content/gdrive')\n",
        "\n",
        "left = h5py.File(\"/content/gdrive/My Drive/Projects/NLP/President Speech/left.hdf5\", \"r\")\n",
        "right = h5py.File(\"/content/gdrive/My Drive/Projects/NLP/President Speech/right.hdf5\", \"r\")\n",
        "out = h5py.File(\"/content/gdrive/My Drive/Projects/NLP/President Speech/out.hdf5\", \"r\")"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8tluT1yLZHgb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_left = np.array(left.get('dataset_left'))\n",
        "X_right = np.array(right.get('dataset_right'))\n",
        "Y = np.array(out.get('dataset_out'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mcVFvZrXPyGy",
        "colab_type": "text"
      },
      "source": [
        "# 3. Network"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YHLpwjwaWpho",
        "colab_type": "text"
      },
      "source": [
        "## 3.1. Network Modelling"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GelyMkYNPni8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 496
        },
        "outputId": "f52d2b08-a510-435a-dfed-e9d3e10fa6c6"
      },
      "source": [
        "inp_left = Input(shape=(X_left.shape[1], X_left.shape[2]), name='input_left')\n",
        "inp_right = Input(shape=(X_right.shape[1], X_right.shape[2]), name='input_right')\n",
        "\n",
        "left = LSTM(100, return_sequences=True, name='lstm_left')(inp_left)\n",
        "right = LSTM(100, return_sequences=True, name='lstm_right')(inp_right)\n",
        "\n",
        "a = concatenate([left, right], axis=2, name='a')\n",
        "flat = Flatten(name='flat')(a)\n",
        "dense = Dense(100, activation='relu', name='dense')(flat)\n",
        "output = Dense(Y.shape[1], activation='softmax')(dense)\n",
        "model = Model(inputs=[inp_left, inp_right], outputs=output)\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
        "model.summary()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Colocations handled automatically by placer.\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_left (InputLayer)         (None, 10, 1)        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_right (InputLayer)        (None, 10, 1)        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "lstm_left (LSTM)                (None, 10, 100)      40800       input_left[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "lstm_right (LSTM)               (None, 10, 100)      40800       input_right[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "a (Concatenate)                 (None, 10, 200)      0           lstm_left[0][0]                  \n",
            "                                                                 lstm_right[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "flat (Flatten)                  (None, 2000)         0           a[0][0]                          \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 100)          200100      flat[0][0]                       \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 8073)         815373      dense[0][0]                      \n",
            "==================================================================================================\n",
            "Total params: 1,097,073\n",
            "Trainable params: 1,097,073\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SmeeosOyWwQt",
        "colab_type": "text"
      },
      "source": [
        "## 3.2. Hyperparameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-lN9aFAgWoKG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "#Xleft_train, Xleft_test, Xright_train, Xright_test, Ytrain, Ytest = train_test_split(X_left, X_right, Y, test_size=0.2, random_state=42)\n",
        "\n",
        "filepath=\"/content/gdrive/My Drive/Projects/NLP/President Speech/weights/weights_bi.hdf5\"\n",
        "\n",
        "callbacks = [\n",
        "    #EarlyStopping(monitor='loss', patience=10, verbose=0, mode='min'),\n",
        "    #ReduceLROnPlateau(monitor='loss', factor=0.1, patience=5, verbose=1, mode='min'),\n",
        "    ModelCheckpoint(filepath, save_best_only=True,  save_weights_only=False, mode='min', verbose=1)\n",
        "]\n",
        "#model.load_weights('/content/gdrive/My Drive/Projects/NLP/weights-improvement-20-1.9923.hdf5')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yu9-d8vuhp0j",
        "colab_type": "text"
      },
      "source": [
        "## 3.3 Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6pgzV2q7adyT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "b5c08746-74b5-43a3-fba8-65c9ca77f22d"
      },
      "source": [
        "model = load_model('/content/gdrive/My Drive/Projects/NLP/President Speech/biLSTM.h5')\n",
        "for i in range(100):\n",
        "    model.fit({'input_left': X_left, 'input_right': X_right}, Y, epochs=10, batch_size=256, callbacks=callbacks)\n",
        "    model.save('/content/gdrive/My Drive/Projects/NLP/President Speech/biLSTM.h5')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "133942/133942 [==============================] - 24s 182us/step - loss: 4.2331\n",
            "Epoch 7/10\n",
            "100096/133942 [=====================>........] - ETA: 6s - loss: 4.2254"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t-dsM8O0z4le",
        "colab_type": "text"
      },
      "source": [
        "# GPU Usage"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CLb6Kw8wyP4l",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "1b980243-9c61-4cb2-9687-70b9ef87ff70"
      },
      "source": [
        "'''!ln -sf /opt/bin/nvidia-smi /usr/bin/nvidia-smi\n",
        "!pip install gputil\n",
        "!pip install psutil\n",
        "!pip install humanize'''\n",
        "import psutil\n",
        "import humanize\n",
        "import os\n",
        "import GPUtil as GPU\n",
        "GPUs = GPU.getGPUs()\n",
        "# XXX: only one GPU on Colab and isn’t guaranteed\n",
        "gpu = GPUs[0]\n",
        "def printm():\n",
        "    process = psutil.Process(os.getpid())\n",
        "    print(\"Gen RAM Free: \" + humanize.naturalsize( psutil.virtual_memory().available ), \" | Proc size: \" + humanize.naturalsize( process.memory_info().rss))\n",
        "    print(\"GPU RAM Free: {0:.0f}MB | Used: {1:.0f}MB | Util {2:3.0f}% | Total {3:.0f}MB\".format(gpu.memoryFree, gpu.memoryUsed, gpu.memoryUtil*100, gpu.memoryTotal))\n",
        "printm()"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Gen RAM Free: 12.7 GB  | Proc size: 581.2 MB\n",
            "GPU RAM Free: 15079MB | Used: 0MB | Util   0% | Total 15079MB\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}